import matplotlib.image as mpimg
import matplotlib.pyplot as plt
import numpy as np
import cv2
%matplotlib inline
from moviepy.editor import VideoFileClip
from IPython.display import HTML
import queue
import pickle
import os
from scipy.ndimage.measurements import label
def visualize_images(input_images, num_cols, figure_name, cmap = None):
"Shows input images by stacking them in num_cols columns"
fig, axes = plt.subplots((int)((len(input_images) + 1) /num_cols), num_cols, figsize=(24, 20))
fig = plt.gcf()
fig.canvas.set_window_title(figure_name)
print(figure_name)
for ax, image in zip(axes.flat, input_images):
if(cmap == "gray" or cmap == 'hot'):
ax.imshow(image, cmap=cmap)
elif(image.shape[2]==1):
ax.imshow(image[:,:,0], cmap = cmap)
else:
ax.imshow(image, cmap=cmap)
plt.show()
import glob
#Read cars and not-cars images
#Data folders
vehicles_dir = './vehicles/'
non_vehicles_dir = './non-vehicles/'
# images are divided up into vehicles and non-vehicles
cars = []
notcars = []
# Read vehicle images
images = glob.iglob(vehicles_dir + '/**/*.png', recursive=True)
for image in images:
cars.append(image)
# Read non-vehicle images
images = glob.iglob(non_vehicles_dir + '/**/*.png', recursive=True)
for image in images:
notcars.append(image)
#Visualize some input images
import random
num_images = 10
# Just for fun choose random car / not-car indices and plot example images
cars_samples = random.sample(list(cars), num_images)
notcar_samples = random.sample(list(notcars), num_images)
# Read in car / not-car images
car_images = []
notcar_images = []
for sample in cars_samples:
car_images.append(mpimg.imread(sample))
for sample in notcar_samples:
notcar_images.append(mpimg.imread(sample))
visualize_images(car_images, num_images, "Example Car images")
visualize_images(notcar_images, num_images, "Example not-car images")
def convert_color(img, conv='RGB2YCrCb'):
"""
Convert the image from one color space to the other
"""
if conv == 'RGB2YCrCb':
return cv2.cvtColor(img, cv2.COLOR_RGB2YCrCb)
if conv == 'BGR2YCrCb':
return cv2.cvtColor(img, cv2.COLOR_BGR2YCrCb)
if conv == 'RGB2LUV':
return cv2.cvtColor(img, cv2.COLOR_RGB2LUV)
if conv == 'RGB2HLS':
return cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
if conv == 'RGB2HSV':
return cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
if conv == 'Gray':
return cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
if conv == 'RGB2YUV':
return cv2.cvtColor(img, cv2.COLOR_RGB2YUV)
# Plot the examples
fig = plt.figure()
plt.subplot(141)
yuv_image = convert_color(car_images[1], 'RGB2YUV')
plt.imshow(yuv_image[:,:,0], cmap ="gray")
plt.title('Y channel')
plt.subplot(142)
plt.imshow(yuv_image[:,:,1], cmap ="gray")
plt.title('U Channel')
plt.subplot(143)
plt.imshow(yuv_image[:,:,2], cmap ="gray")
plt.title('V Channel')
plt.subplot(144)
plt.imshow(yuv_image)
plt.title('YUV')
from skimage.feature import hog
def get_hog_features(img, orient, pix_per_cell, cell_per_block,
vis=False, feature_vec=True):
"""
Return the hog features of the given input image
Call with two outputs if vis==True"""
if vis == True:
features, hog_image = hog(img, orientations=orient, pixels_per_cell=(pix_per_cell, pix_per_cell),
cells_per_block=(cell_per_block, cell_per_block), transform_sqrt=True,
visualize=vis, feature_vector=feature_vec)
return features, hog_image
# Otherwise call with one output
else:
features = hog(img, orientations=orient, pixels_per_cell=(pix_per_cell, pix_per_cell),
cells_per_block=(cell_per_block, cell_per_block), transform_sqrt=True,
visualize=vis, feature_vector=feature_vec)
return features
orient = 9
pix_per_cell = 8 #8*8
cell_per_block = 2 #2*2
#notes:
#get_hog_features() only accepts greyscale images
#else we will calculate hog for every channel R G B
car_features, hog_image = get_hog_features(cv2.cvtColor(car_images[1], cv2.COLOR_RGB2GRAY), orient, pix_per_cell, cell_per_block,
vis=True, feature_vec=True)
notcar_features, notcar_hog_image = get_hog_features(cv2.cvtColor(notcar_images[2], cv2.COLOR_RGB2GRAY), orient, pix_per_cell, cell_per_block,
vis=True, feature_vec=True)
# Plot the examples
fig = plt.figure()
plt.subplot(131)
plt.imshow(car_images[1])
plt.title('Car Image')
plt.subplot(132)
plt.imshow(hog_image, cmap='gray')
plt.title('car HOG')
plt.subplot(133)
plt.imshow(notcar_hog_image, cmap='gray')
plt.title('not car HOG')
# While it could be heavy to include three color channels of a full resolution image,
# you can perform spatial binning on an image and still retain enough information to help in finding vehicles.
# even going all the way down to 32 x 32 pixel resolution,
# the car itself is still clearly identifiable by eye, and this means that the relevant features are still preserved at this resolution.
def bin_spatial(img, size=(16, 16)): # to resize image combine each 4 pixel into 1 to make use of color info as a feature
color1 = cv2.resize(img[:,:,0], size).ravel()
color2 = cv2.resize(img[:,:,1], size).ravel()
color3 = cv2.resize(img[:,:,2], size).ravel()
return np.hstack((color1, color2, color3))
# Define a function to compute color histogram features
# Compute the histogram of the RGB channels separately
# Concatenate the histograms into a single feature vector
# NEED TO CHANGE bins_range if reading .png files with mpimg! 32 is the width for 1 region in histogram
def color_hist(img, nbins=32, bins_range=(0, 256)): # to normalize histogram 1 Byte for pixel ranges [0-256] 256/32 = 8 --> 8 regions for histogram
# Compute the histogram of the color channels separately
channel1_hist = np.histogram(img[:,:,0], bins=nbins, range=bins_range)
channel2_hist = np.histogram(img[:,:,1], bins=nbins, range=bins_range)
channel3_hist = np.histogram(img[:,:,2], bins=nbins, range=bins_range)
# Concatenate the histograms into a single feature vector
hist_features = np.concatenate((channel1_hist[0], channel2_hist[0], channel3_hist[0]))
# Return the individual histograms, bin_centers and feature vector
return hist_features
# Define a function to extract features from a list of images
# Have this function call bin_spatial() and color_hist()
def extract_features(imgs, cspace='RGB', spatial_size=(32, 32),
hist_bins=32, orient=9,
pix_per_cell=8, cell_per_block=2, hog_channel='ALL',
spatial_feat=False, hist_feat=False, hog_feat=True):
# Create a list to append feature vectors to
features = []
# Iterate through the list of images
for file in imgs:
file_features = []
# Read in each one by one
image = mpimg.imread(file)
image = (image * 255).astype(np.uint8)
# apply color conversion if other than 'RGB'
if cspace != 'RGB':
if cspace == 'HSV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
elif cspace == 'LUV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)
elif cspace == 'HLS':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)
elif cspace == 'YUV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
elif cspace == 'YCrCb':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YCrCb)
else: feature_image = np.copy(image)
if spatial_feat == True:
spatial_features = bin_spatial(feature_image, size=spatial_size)
file_features.append(spatial_features)
if hist_feat == True:
# Apply color_hist()
hist_features = color_hist(feature_image, nbins=hist_bins)
file_features.append(hist_features)
if hog_feat == True:
# Call get_hog_features() with vis=False, feature_vec=True
hog_features = []
# get hog features for the 3 channels
for channel in range(feature_image.shape[2]):
hog_features.append(get_hog_features(feature_image[:,:,channel],
orient, pix_per_cell, cell_per_block,
vis=False, feature_vec=True))
hog_features = np.ravel(hog_features)
# Append the new feature vector to the features list
file_features.append(hog_features)
features.append(np.concatenate(file_features))
# Return list of feature vectors
return features
import time
from sklearn.svm import LinearSVC
from sklearn.model_selection import train_test_split
from sklearn.decomposition import PCA
from sklearn.utils import shuffle
from sklearn.preprocessing import StandardScaler
colorspace = 'YUV' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 9
pix_per_cell = 8
cell_per_block = 2
hog_channel = 'ALL' # Can be 0, 1, 2, or "ALL"
spatial_size=(32, 32)
hist_bins=32
t=time.time()
car_features = extract_features(cars, cspace=colorspace, orient=orient,
pix_per_cell=pix_per_cell, cell_per_block=cell_per_block,
hog_channel=hog_channel, hist_bins=hist_bins)
notcar_features = extract_features(notcars, cspace=colorspace, orient=orient,
pix_per_cell=pix_per_cell, cell_per_block=cell_per_block,
hog_channel=hog_channel, hist_bins=hist_bins)
t2 = time.time()
print(round(t2-t, 2), 'Seconds to extract HOG features...')
# Create an array stack of feature vectors
X = np.vstack((car_features, notcar_features)).astype(np.float64)
print(X.shape)
# StandardScaler performs the task of Standardization.
# Usually a dataset contains variables that are different in scale.
# For e.g. an Employee dataset will contain AGE column with values on scale 20-70 and SALARY column with values on scale 10000-80000.
# As these two columns are different in scale, they are Standardized to have common scale while building machine learning model.
#normalize the scale
# Fit a per-column scaler
X_scaler = StandardScaler().fit(X)
# Apply the scaler to X
scaled_X = X_scaler.transform(X) #to scale training and test data
# Define the labels vector
y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features))))
print(len(y))
# Split up data into randomized training and test sets
rand_state = np.random.randint(0, 100)
X_train, X_test, y_train, y_test = train_test_split(
scaled_X, y, test_size=0.15, random_state=rand_state)
#X_train, X_test = shuffle(X_train, y_train, random_state=rand_state)
# # Compute a PCA on the features
# n_components = 4932
# print("Extracting the top %d features from %d total features"
# % (n_components, X_train.shape[1]))
# pca = PCA(n_components=n_components, svd_solver='randomized',
# whiten=True).fit(X_train)
# X_train_pca = pca.transform(X_train)
print('Using:',orient,'orientations',pix_per_cell,
'pixels per cell and', cell_per_block,'cells per block')
print('Feature vector length:', len(X_train[0]))
# Use a linear SVC X_scaler
svc = LinearSVC()
# Check the training time for the SVC
t=time.time()
svc.fit(X_train, y_train)
t2 = time.time()
print(round(t2-t, 2), 'Seconds to train SVC...')
# Check the score of the SVC
print('Test Accuracy of SVC = ', round(svc.score(X_test, y_test), 4))
# Check the prediction time for a single sample
t=time.time()
n_predict = 10
print('My SVC predicts: ', svc.predict(X_test[0:n_predict]))
print('For these',n_predict, 'labels: ', y_test[0:n_predict])
t2 = time.time()
print(round(t2-t, 5), 'Seconds to predict', n_predict,'labels with SVC')
#Pickle the data as it takes a lot of time to generate it
pickle.dump(svc, open('svc_pickle.pkl', "wb"))
pickle.dump(X_scaler, open('X_scaler.pkl', "wb"))
#Read cars and not-cars images
#Data folders
test_images_dir = './test_images/'
# images are divided up into vehicles and non-vehicles
test_images = []
images = glob.glob(test_images_dir + '*.jpg')
for image in images:
test_images.append(mpimg.imread(image))
# Define a single function that can extract features using hog sub-sampling and make predictions
def find_cars(img, ystart, ystop, scale, svc, X_scaler, orient, pix_per_cell, cell_per_block, spatial_size, hist_bins, vis_bboxes = False):
draw_img = np.copy(img)
xstart = int(img.shape[1]/5) #from 1/5 of image
xstop = img.shape[1] #till end of image
img_tosearch = img[ystart:ystop, xstart:xstop,:] #crop image --> make it = size of rectangle
ctrans_tosearch = convert_color(img_tosearch, conv='RGB2YUV')
if scale != 1:
imshape = ctrans_tosearch.shape
ctrans_tosearch = cv2.resize(ctrans_tosearch, (np.int(imshape[1]/scale), np.int(imshape[0]/scale)))
ch1 = ctrans_tosearch[:,:,0]
ch2 = ctrans_tosearch[:,:,1]
ch3 = ctrans_tosearch[:,:,2]
# Define blocks and steps as above
nxblocks = (ch1.shape[1] // pix_per_cell) - cell_per_block + 1
nyblocks = (ch1.shape[0] // pix_per_cell) - cell_per_block + 1
nfeat_per_block = orient*cell_per_block**2
# 64 was the orginal sampling rate, with 8 cells and 8 pix per cell
window = 64
nblocks_per_window = (window // pix_per_cell) - cell_per_block + 1
cells_per_step = 2 # Instead of overlap, define how many cells to step
nxsteps = (nxblocks - nblocks_per_window) // cells_per_step
nysteps = (nyblocks - nblocks_per_window) // cells_per_step
# Compute individual channel HOG features for the entire image
hog1 = get_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False)
hog2 = get_hog_features(ch2, orient, pix_per_cell, cell_per_block, feature_vec=False)
hog3 = get_hog_features(ch3, orient, pix_per_cell, cell_per_block, feature_vec=False)
rectangles = []
for xb in range(nxsteps):
for yb in range(nysteps):
ypos = yb*cells_per_step
xpos = xb*cells_per_step
# Extract HOG for this patch
hog_feat1 = hog1[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() # ravel flaten the array
hog_feat2 = hog2[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
hog_feat3 = hog3[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
hog_features = np.hstack((hog_feat1, hog_feat2, hog_feat3)).reshape(1, -1)
# hog_features = np.hstack((hog_feat1))
xleft = xpos*pix_per_cell
ytop = ypos*pix_per_cell
# Extract the image patch
# subimg = cv2.resize(ctrans_tosearch[ytop:ytop+window, xleft:xleft+window], (64,64))
# Get color features
# spatial_features = bin_spatial(subimg, size=spatial_size)
# hist_features = color_hist(subimg, nbins=hist_bins)
# Scale features and make a prediction
# stacked = np.hstack((spatial_features, hist_features, hog_features)).reshape(1, -1)
test_features = X_scaler.transform(hog_features)
#test_features = X_scaler.transform(np.hstack((shape_feat, hist_feat)).reshape(1, -1))
test_prediction = svc.predict(test_features)
if test_prediction == 1 or vis_bboxes == True:
xbox_left = np.int(xleft*scale)
ytop_draw = np.int(ytop*scale)
win_draw = np.int(window*scale)
rectangles.append(((xbox_left+xstart, ytop_draw+ystart),(xbox_left+win_draw+xstart,ytop_draw+win_draw+ystart)))
return rectangles
def get_rectangles(image, scales = [1, 1.5, 2, 2.5, 3],
ystarts = [400, 400, 450, 450, 460],
ystops = [528, 550, 620, 650, 700]):
out_rectangles = []
X_scaler = pickle.load(open('X_scaler.pkl', 'rb'))
svc = pickle.load(open('svc_pickle.pkl', 'rb'))
spatial_size = 32
hist_bins = 32
for scale, ystart, ystop in zip(scales, ystarts, ystops): #zip [(1, 400, 528), (1.5, 400, 550), ...]
rectangles = find_cars(image, ystart, ystop, scale, svc, X_scaler, orient, pix_per_cell, cell_per_block, spatial_size, hist_bins)
if len(rectangles) > 0:
out_rectangles.append(rectangles)
out_rectangles = [item for sublist in out_rectangles for item in sublist]
return out_rectangles
def add_heat(heatmap, bbox_list):
# Iterate through list of bboxes
for box in bbox_list:
# Add += 1 for all pixels inside each bbox
# Assuming each "box" takes the form ((x1, y1), (x2, y2))
heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1
# Return updated heatmap
return heatmap
def apply_threshold(heatmap, threshold):
# Zero out pixels below the threshold
heatmap[heatmap <= threshold] = 0
# Return thresholded map
return heatmap
def draw_labeled_bboxes(img, labels):
# Iterate through all detected cars
img_copy = np.copy(img)
result_rectangles = []
for car_number in range(1, labels[1]+1):
# Find pixels with each car_number label value
nonzero = (labels[0] == car_number).nonzero()
# Identify x and y values of those pixels
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
# Define a bounding box based on min/max x and y
bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))
area = (bbox[1][1] - bbox[0][1]) * (bbox[1][0] - bbox[0][0])
if area > 40 * 40:
result_rectangles.append(bbox)
# Draw the box on the image
# cv2.rectangle(img_copy, bbox[0], bbox[1], (0,255,0), 6) (bbox[0][0]-100,bbox[0][1]-100), (bbox[1][0]+100,bbox[1][1]+100)
cv2.rectangle(img_copy,(bbox[0][0]-int(0.02*bbox[0][0]),bbox[0][1]-int(0.02*bbox[0][1])), (bbox[1][0]+int(0.02*bbox[1][0]),bbox[1][1]+int(0.02*bbox[1][1])), (0,255,0), 6)
# Return the image
return result_rectangles, img_copy
def draw_boxes(img, bboxes, color=(0, 0, 255), thick=6):
# Make a copy of the image
imcopy = np.copy(img)
random_color = False
# Iterate through the bounding boxes
for bbox in bboxes:
if color == 'random' or random_color:
color = (np.random.randint(0,255), np.random.randint(0,255), np.random.randint(0,255))
random_color = True
# Draw a rectangle given bbox coordinates
cv2.rectangle(imcopy, bbox[0], bbox[1], color, thick)
# cv2.rectangle(imcopy, (bbox[0][0]-100,bbox[0][1]-100), (bbox[1][0]+100,bbox[1][1]+100), color, thick)
# Return the image copy with boxes drawn
return imcopy
def visualize_bboxes(image, scales = [1, 1.5, 2, 2.5, 3],
ystarts = [400, 400, 450, 450, 460],
ystops = [528, 550, 620, 650, 700]):
out_rectangles = []
svc = pickle.load(open('svc_pickle.pkl', 'rb'))
for scale, ystart, ystop in zip(scales, ystarts, ystops):
rectangles = find_cars(image, ystart, ystop, scale, svc, X_scaler, orient, pix_per_cell, cell_per_block, spatial_size, hist_bins, vis_bboxes = True)
if len(rectangles) > 0:
out_rectangles.append(rectangles)
out_rectangles = [item for sublist in out_rectangles for item in sublist]
plt.figure(figsize=(20,10))
plt.imshow(draw_boxes(image, out_rectangles, color='random', thick=3))
from scipy.ndimage.measurements import label
result_images = []
result_boxes = []
heatmap_images = []
result_img_all_boxes = []
for test_image in test_images:
rectangles = get_rectangles(test_image) # get all rectangles with coordinates --> (top left corner, bottom right corner)
result_img_all_boxes.append(draw_boxes(test_image, rectangles, color='random', thick=3)) # draw all rectangles with different colors
heatmap_image = np.zeros_like(test_image[:, :, 0])
heatmap_image = add_heat(heatmap_image, rectangles) #bbt3at numpy array of zeros wel return byb2a (Add += 1 for all pixels inside each bbox) y3ni ha5od el gowa el boxes bs ba2i el sora b zeros
heatmap_images.append(heatmap_image)
heatmap_image = apply_threshold(heatmap_image, 2)
labels = label(heatmap_image)
rectangles, result_image = draw_labeled_bboxes(test_image, labels)
result_boxes.append(rectangles)
result_images.append(result_image)
visualize_images(result_img_all_boxes, 2, "test")
visualize_images(heatmap_images, 2, "Heatmap", cmap="hot")
visualize_images(result_images, 2, "test")
class DetectionInfo():
def __init__(self):
self.max_size = 10
self.old_bboxes = queue.Queue(self.max_size)
self.heatmap = np.zeros_like(test_images[0][:, :, 0])
#added
def draw_heat_map(self, bboxes):
heat_img = np.zeros_like(test_images[0][:, :, 0])
heat_img = add_heat(heat_img, bboxes) #bbt3at numpy array of zeros wel return byb2a (Add += 1 for all pixels inside each bbox) y3ni ha5od el gowa el boxes bs ba2i el sora b zeros
return heat_img
#added
def draw_all_boxes(self, image, bboxes):
all_boxes = draw_boxes(image, bboxes, color='random', thick=3)
return all_boxes
def get_heatmap(self):
self.heatmap = np.zeros_like(test_images[0][:, :, 0])
if self.old_bboxes.qsize() == self.max_size:
for bboxes in list(self.old_bboxes.queue):
self.heatmap = add_heat(self.heatmap, bboxes)
self.heatmap = apply_threshold(self.heatmap, 20)
return self.heatmap
def get_labels(self):
return label(self.get_heatmap())
def add_bboxes(self, bboxes):
if len(bboxes) < 1:
return
if self.old_bboxes.qsize() == self.max_size:
self.old_bboxes.get()
self.old_bboxes.put(bboxes)
def combine_images( big_image, small1, small2):
background = np.zeros_like(big_image)
large_img_size = (background.shape[1],int(0.75*background.shape[0]))
small_img_size=(int(background.shape[1]/2),int(0.25*background.shape[0]))
small_img_x_offset=0
small_img_y_offset=0
img2 = cv2.resize(np.dstack((small2, small2, small2))*255, small_img_size)
img1 = cv2.resize(small1, small_img_size)
img3 = cv2.resize(big_image, large_img_size)
background[0: small_img_size[1], 0: small_img_size[0]] = img2
start_offset_y = small_img_y_offset
endy = start_offset_y + small_img_size[1]
if endy > background.shape[0]:
endy = background.shape[0]
start_offset_x = 2 * small_img_x_offset + small_img_size[0]
endx = start_offset_x + small_img_size[0]
if endx > background.shape[1]:
endx = background.shape[1]
background[start_offset_y:endy , start_offset_x: endx] = img1
background[int(0.25*background.shape[0]): , : ] = img3
return background
def find_vehicles(image):
bboxes = get_rectangles(image)
detection_info.add_bboxes(bboxes)
labels = detection_info.get_labels()
if len(labels) == 0:
result_image = image
else:
bboxes, result_image = draw_labeled_bboxes(image,labels)
return result_image
def find_vehicles_debug(image):
out_img = np.copy(image)
bboxes = get_rectangles(image)
detection_info.add_bboxes(bboxes)
all_boxes = detection_info.draw_all_boxes(out_img, bboxes) ###########
heat_img = detection_info.draw_heat_map(bboxes) ############
labels = detection_info.get_labels()
if len(labels) == 0:
result_image = image
else:
bboxes, result_image = draw_labeled_bboxes(image,labels)
result = combine_images(result_image, all_boxes, heat_img)
return result
detection_info = DetectionInfo()
detection_info.old_heatmap = np.zeros_like(test_images[0][:, :, 0])
project_video_path = './trimmed.mp4'
project_video_output = './out_test.mp4'
project_video = VideoFileClip(project_video_path)
white_clip = project_video.fl_image(find_vehicles) #NOTE: this function expects color images!!
%time white_clip.write_videofile(project_video_output, audio=False)
!pip install nbconvert
!jupyter nbconvert --to html final_code.ipynb